{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import * \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checking_balance</th>\n",
       "      <th>months_loan_duration</th>\n",
       "      <th>credit_history</th>\n",
       "      <th>purpose</th>\n",
       "      <th>amount</th>\n",
       "      <th>savings_balance</th>\n",
       "      <th>employment_length</th>\n",
       "      <th>installment_rate</th>\n",
       "      <th>personal_status</th>\n",
       "      <th>other_debtors</th>\n",
       "      <th>...</th>\n",
       "      <th>property</th>\n",
       "      <th>age</th>\n",
       "      <th>installment_plan</th>\n",
       "      <th>housing</th>\n",
       "      <th>existing_credits</th>\n",
       "      <th>default</th>\n",
       "      <th>dependents</th>\n",
       "      <th>telephone</th>\n",
       "      <th>foreign_worker</th>\n",
       "      <th>job</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>6</td>\n",
       "      <td>critical</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>1169</td>\n",
       "      <td>unknown</td>\n",
       "      <td>&gt; 7 yrs</td>\n",
       "      <td>4</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>67</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 - 200 DM</td>\n",
       "      <td>48</td>\n",
       "      <td>repaid</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>5951</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>22</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unknown</td>\n",
       "      <td>12</td>\n",
       "      <td>critical</td>\n",
       "      <td>education</td>\n",
       "      <td>2096</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>49</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>unskilled resident</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>42</td>\n",
       "      <td>repaid</td>\n",
       "      <td>furniture</td>\n",
       "      <td>7882</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>guarantor</td>\n",
       "      <td>...</td>\n",
       "      <td>building society savings</td>\n",
       "      <td>45</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>24</td>\n",
       "      <td>delayed</td>\n",
       "      <td>car (new)</td>\n",
       "      <td>4870</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>3</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>unknown/none</td>\n",
       "      <td>53</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  checking_balance  months_loan_duration credit_history    purpose  amount  \\\n",
       "0           < 0 DM                     6       critical   radio/tv    1169   \n",
       "1       1 - 200 DM                    48         repaid   radio/tv    5951   \n",
       "2          unknown                    12       critical  education    2096   \n",
       "3           < 0 DM                    42         repaid  furniture    7882   \n",
       "4           < 0 DM                    24        delayed  car (new)    4870   \n",
       "\n",
       "  savings_balance employment_length  installment_rate personal_status  \\\n",
       "0         unknown           > 7 yrs                 4     single male   \n",
       "1        < 100 DM         1 - 4 yrs                 2          female   \n",
       "2        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "3        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "4        < 100 DM         1 - 4 yrs                 3     single male   \n",
       "\n",
       "  other_debtors  ...                  property age  installment_plan  \\\n",
       "0          none  ...               real estate  67              none   \n",
       "1          none  ...               real estate  22              none   \n",
       "2          none  ...               real estate  49              none   \n",
       "3     guarantor  ...  building society savings  45              none   \n",
       "4          none  ...              unknown/none  53              none   \n",
       "\n",
       "    housing existing_credits  default  dependents  telephone foreign_worker  \\\n",
       "0       own                2        1           1        yes            yes   \n",
       "1       own                1        2           1       none            yes   \n",
       "2       own                1        1           2       none            yes   \n",
       "3  for free                1        1           2       none            yes   \n",
       "4  for free                2        2           2       none            yes   \n",
       "\n",
       "                  job  \n",
       "0    skilled employee  \n",
       "1    skilled employee  \n",
       "2  unskilled resident  \n",
       "3    skilled employee  \n",
       "4    skilled employee  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/data/credit-default.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 21 columns):\n",
      "checking_balance        1000 non-null object\n",
      "months_loan_duration    1000 non-null int64\n",
      "credit_history          1000 non-null object\n",
      "purpose                 1000 non-null object\n",
      "amount                  1000 non-null int64\n",
      "savings_balance         1000 non-null object\n",
      "employment_length       1000 non-null object\n",
      "installment_rate        1000 non-null int64\n",
      "personal_status         1000 non-null object\n",
      "other_debtors           1000 non-null object\n",
      "residence_history       1000 non-null int64\n",
      "property                1000 non-null object\n",
      "age                     1000 non-null int64\n",
      "installment_plan        1000 non-null object\n",
      "housing                 1000 non-null object\n",
      "existing_credits        1000 non-null int64\n",
      "default                 1000 non-null int64\n",
      "dependents              1000 non-null int64\n",
      "telephone               1000 non-null object\n",
      "foreign_worker          1000 non-null object\n",
      "job                     1000 non-null object\n",
      "dtypes: int64(8), object(13)\n",
      "memory usage: 164.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    700\n",
       "2    300\n",
       "Name: default, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.default.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checking_balance</th>\n",
       "      <th>months_loan_duration</th>\n",
       "      <th>credit_history</th>\n",
       "      <th>purpose</th>\n",
       "      <th>amount</th>\n",
       "      <th>savings_balance</th>\n",
       "      <th>employment_length</th>\n",
       "      <th>installment_rate</th>\n",
       "      <th>personal_status</th>\n",
       "      <th>other_debtors</th>\n",
       "      <th>residence_history</th>\n",
       "      <th>property</th>\n",
       "      <th>age</th>\n",
       "      <th>installment_plan</th>\n",
       "      <th>housing</th>\n",
       "      <th>existing_credits</th>\n",
       "      <th>dependents</th>\n",
       "      <th>telephone</th>\n",
       "      <th>foreign_worker</th>\n",
       "      <th>job</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>6</td>\n",
       "      <td>critical</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>1169</td>\n",
       "      <td>unknown</td>\n",
       "      <td>&gt; 7 yrs</td>\n",
       "      <td>4</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>4</td>\n",
       "      <td>real estate</td>\n",
       "      <td>67</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 - 200 DM</td>\n",
       "      <td>48</td>\n",
       "      <td>repaid</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>5951</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>female</td>\n",
       "      <td>none</td>\n",
       "      <td>2</td>\n",
       "      <td>real estate</td>\n",
       "      <td>22</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unknown</td>\n",
       "      <td>12</td>\n",
       "      <td>critical</td>\n",
       "      <td>education</td>\n",
       "      <td>2096</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>3</td>\n",
       "      <td>real estate</td>\n",
       "      <td>49</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>unskilled resident</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>42</td>\n",
       "      <td>repaid</td>\n",
       "      <td>furniture</td>\n",
       "      <td>7882</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>4 - 7 yrs</td>\n",
       "      <td>2</td>\n",
       "      <td>single male</td>\n",
       "      <td>guarantor</td>\n",
       "      <td>4</td>\n",
       "      <td>building society savings</td>\n",
       "      <td>45</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt; 0 DM</td>\n",
       "      <td>24</td>\n",
       "      <td>delayed</td>\n",
       "      <td>car (new)</td>\n",
       "      <td>4870</td>\n",
       "      <td>&lt; 100 DM</td>\n",
       "      <td>1 - 4 yrs</td>\n",
       "      <td>3</td>\n",
       "      <td>single male</td>\n",
       "      <td>none</td>\n",
       "      <td>4</td>\n",
       "      <td>unknown/none</td>\n",
       "      <td>53</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>skilled employee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  checking_balance  months_loan_duration credit_history    purpose  amount  \\\n",
       "0           < 0 DM                     6       critical   radio/tv    1169   \n",
       "1       1 - 200 DM                    48         repaid   radio/tv    5951   \n",
       "2          unknown                    12       critical  education    2096   \n",
       "3           < 0 DM                    42         repaid  furniture    7882   \n",
       "4           < 0 DM                    24        delayed  car (new)    4870   \n",
       "\n",
       "  savings_balance employment_length  installment_rate personal_status  \\\n",
       "0         unknown           > 7 yrs                 4     single male   \n",
       "1        < 100 DM         1 - 4 yrs                 2          female   \n",
       "2        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "3        < 100 DM         4 - 7 yrs                 2     single male   \n",
       "4        < 100 DM         1 - 4 yrs                 3     single male   \n",
       "\n",
       "  other_debtors  residence_history                  property  age  \\\n",
       "0          none                  4               real estate   67   \n",
       "1          none                  2               real estate   22   \n",
       "2          none                  3               real estate   49   \n",
       "3     guarantor                  4  building society savings   45   \n",
       "4          none                  4              unknown/none   53   \n",
       "\n",
       "  installment_plan   housing  existing_credits  dependents telephone  \\\n",
       "0             none       own                 2           1       yes   \n",
       "1             none       own                 1           1      none   \n",
       "2             none       own                 1           2      none   \n",
       "3             none  for free                 1           2      none   \n",
       "4             none  for free                 2           2      none   \n",
       "\n",
       "  foreign_worker                 job  \n",
       "0            yes    skilled employee  \n",
       "1            yes    skilled employee  \n",
       "2            yes  unskilled resident  \n",
       "3            yes    skilled employee  \n",
       "4            yes    skilled employee  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = \"default\"\n",
    "label_encoder = preprocessing.LabelEncoder()\n",
    "\n",
    "y = label_encoder.fit_transform(df[target])\n",
    "X = df.drop(columns=[target])\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['checking_balance',\n",
       " 'credit_history',\n",
       " 'purpose',\n",
       " 'savings_balance',\n",
       " 'employment_length',\n",
       " 'personal_status',\n",
       " 'other_debtors',\n",
       " 'property',\n",
       " 'installment_plan',\n",
       " 'housing',\n",
       " 'telephone',\n",
       " 'foreign_worker',\n",
       " 'job']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_columns = [field for field in dict(X.dtypes) \n",
    "               if X.dtypes[field] == \"object\"]\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['months_loan_duration',\n",
       " 'amount',\n",
       " 'installment_rate',\n",
       " 'residence_history',\n",
       " 'age',\n",
       " 'existing_credits',\n",
       " 'dependents']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_columns = [field for field in dict(X.dtypes) \n",
    "               if X.dtypes[field] != \"object\"]\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_pipe = pipeline.Pipeline([\n",
    "    ('imputer', impute.SimpleImputer(strategy='constant'\n",
    "                                     , fill_value='missing')),\n",
    "    ('onehot', preprocessing.OneHotEncoder(handle_unknown='error'\n",
    "                                           , drop=\"first\"))\n",
    "]) \n",
    "\n",
    "num_pipe = pipeline.Pipeline([\n",
    "    ('imputer', impute.SimpleImputer(strategy='median')),\n",
    "    ('poly', preprocessing.PolynomialFeatures(degree=1\n",
    "                                    , include_bias=False)),\n",
    "    ('scaler', preprocessing.StandardScaler()),\n",
    "])\n",
    "\n",
    "preprocessing_pipe = compose.ColumnTransformer([\n",
    "    (\"cat\", cat_pipe, cat_columns),\n",
    "    (\"num\", num_pipe, num_columns)\n",
    "])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score:  0.755 Best parameters:  {'est__C': 1.557903180725127}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done  35 out of  50 | elapsed:    1.5s remaining:    0.7s\n",
      "[Parallel(n_jobs=8)]: Done  50 out of  50 | elapsed:    1.6s finished\n"
     ]
    }
   ],
   "source": [
    "estimator_pipe = pipeline.Pipeline([\n",
    "    (\"preprocessing\", preprocessing_pipe),\n",
    "    (\"est\", linear_model.LogisticRegression(random_state=1\n",
    "                                , solver=\"liblinear\"))\n",
    "])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"est__C\": np.random.random(10) + 1\n",
    "}\n",
    "\n",
    "gsearch = model_selection.GridSearchCV(estimator_pipe, param_grid, cv = 5\n",
    "                        , verbose=1, n_jobs=8, scoring=\"accuracy\")\n",
    "\n",
    "gsearch.fit(X, y)\n",
    "\n",
    "print(\"Best score: \", gsearch.best_score_\n",
    "        , \"Best parameters: \", gsearch.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "Best score:  0.765 Best parameters:  {'est__svm__C': 5.222222222222222}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done  50 out of  50 | elapsed:    1.1s finished\n"
     ]
    }
   ],
   "source": [
    "log_clf = linear_model.LogisticRegression(C = 1.53\n",
    "                            , solver= \"liblinear\", random_state=1) \n",
    "rnd_clf = ensemble.RandomForestClassifier(max_depth=6\n",
    "                            , n_estimators = 30, random_state=1) \n",
    "svm_clf = svm.SVC(C = 1.0, gamma = 0.15, random_state=1) \n",
    "\n",
    "\n",
    "estimator_pipe = pipeline.Pipeline([\n",
    "    (\"preprocessing\", preprocessing_pipe),\n",
    "    (\"est\", ensemble.VotingClassifier(voting=\"hard\", estimators=\n",
    "                                      [('lr', log_clf), \n",
    "                                       ('rf', rnd_clf), \n",
    "                                       ('svm', svm_clf)\n",
    "                                      ])\n",
    "    )\n",
    "])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"est__svm__C\": np.linspace(1.0, 20, 10)\n",
    "}\n",
    "\n",
    "gsearch = model_selection.GridSearchCV(estimator_pipe, param_grid, cv = 5\n",
    "                    , verbose=1, n_jobs=8, scoring=\"accuracy\")\n",
    "\n",
    "gsearch.fit(X, y)\n",
    "\n",
    "print(\"Best score: \", gsearch.best_score_, \"Best parameters: \", gsearch.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaBoost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    2.9s\n",
      "[Parallel(n_jobs=8)]: Done  50 out of  50 | elapsed:    3.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score:  0.734 Best parameters:  {'est__base_estimator__C': 1.0258494070869997}\n"
     ]
    }
   ],
   "source": [
    "estimator_pipe = pipeline.Pipeline([\n",
    "    (\"preprocessing\", preprocessing_pipe),\n",
    "    (\"est\", ensemble.AdaBoostClassifier(\n",
    "          linear_model.LogisticRegression(random_state=1\n",
    "                                          , solver=\"liblinear\")\n",
    "        , n_estimators=200\n",
    "        , algorithm=\"SAMME.R\"\n",
    "        , learning_rate=0.051)\n",
    "\n",
    "    )\n",
    "])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"est__base_estimator__C\": np.random.random(10) + 1\n",
    "}\n",
    "\n",
    "gsearch = model_selection.GridSearchCV(estimator_pipe, param_grid, cv = 5\n",
    "                                , verbose=1, n_jobs=8, scoring=\"accuracy\")\n",
    "\n",
    "gsearch.fit(X, y)\n",
    "\n",
    "print(\"Best score: \", gsearch.best_score_, \"Best parameters: \", gsearch.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score:  0.757 Best parameters:  {'est__base_estimator__max_depth': 12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done  50 out of  50 | elapsed:    1.1s finished\n"
     ]
    }
   ],
   "source": [
    "estimator_pipe = pipeline.Pipeline([\n",
    "    (\"preprocessing\", preprocessing_pipe),\n",
    "    (\"est\", ensemble.BaggingClassifier(\n",
    "                tree.DecisionTreeClassifier(), \n",
    "                max_samples= 0.5,\n",
    "                n_estimators=50,\n",
    "                bootstrap=True, \n",
    "                oob_score=True)\n",
    "    )\n",
    "])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"est__base_estimator__max_depth\": np.arange(5, 15)\n",
    "}\n",
    "\n",
    "gsearch = model_selection.GridSearchCV(estimator_pipe, param_grid, cv = 5\n",
    "                        , verbose=1, n_jobs=8, scoring=\"accuracy\")\n",
    "\n",
    "gsearch.fit(X, y)\n",
    "\n",
    "print(\"Best score: \", gsearch.best_score_, \"Best parameters: \", gsearch.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosted Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 70 candidates, totalling 350 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  52 tasks      | elapsed:    4.5s\n",
      "[Parallel(n_jobs=8)]: Done 205 tasks      | elapsed:   14.8s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score:  0.76 Best parameters:  {'est__learning_rate': 0.12, 'est__max_depth': 3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Done 350 out of 350 | elapsed:   20.8s finished\n"
     ]
    }
   ],
   "source": [
    "estimator_pipe = pipeline.Pipeline([\n",
    "    (\"preprocessing\", preprocessing_pipe),\n",
    "    (\"est\", ensemble.GradientBoostingClassifier(random_state=1))\n",
    "])\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"est__max_depth\": np.arange(3, 10),\n",
    "    \"est__learning_rate\": np.linspace(0.01, 1, 10)\n",
    "}\n",
    "\n",
    "gsearch = model_selection.GridSearchCV(estimator_pipe, param_grid, cv = 5\n",
    "                        , verbose=1, n_jobs=8, scoring=\"accuracy\")\n",
    "\n",
    "gsearch.fit(X, y)\n",
    "\n",
    "print(\"Best score: \", gsearch.best_score_\n",
    "        , \"Best parameters: \", gsearch.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_est__learning_rate</th>\n",
       "      <th>param_est__max_depth</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.178265</td>\n",
       "      <td>0.008670</td>\n",
       "      <td>0.006912</td>\n",
       "      <td>0.001198</td>\n",
       "      <td>0.01</td>\n",
       "      <td>3</td>\n",
       "      <td>{'est__learning_rate': 0.01, 'est__max_depth': 3}</td>\n",
       "      <td>0.705</td>\n",
       "      <td>0.705</td>\n",
       "      <td>0.710</td>\n",
       "      <td>0.685</td>\n",
       "      <td>0.705</td>\n",
       "      <td>0.702</td>\n",
       "      <td>0.008718</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.315358</td>\n",
       "      <td>0.017413</td>\n",
       "      <td>0.006609</td>\n",
       "      <td>0.001425</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4</td>\n",
       "      <td>{'est__learning_rate': 0.01, 'est__max_depth': 4}</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.710</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.014142</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.482575</td>\n",
       "      <td>0.019085</td>\n",
       "      <td>0.008350</td>\n",
       "      <td>0.000537</td>\n",
       "      <td>0.01</td>\n",
       "      <td>5</td>\n",
       "      <td>{'est__learning_rate': 0.01, 'est__max_depth': 5}</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.715</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.011402</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.660290</td>\n",
       "      <td>0.010609</td>\n",
       "      <td>0.006561</td>\n",
       "      <td>0.001300</td>\n",
       "      <td>0.01</td>\n",
       "      <td>6</td>\n",
       "      <td>{'est__learning_rate': 0.01, 'est__max_depth': 6}</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.775</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.730</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.019849</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.838601</td>\n",
       "      <td>0.033753</td>\n",
       "      <td>0.006765</td>\n",
       "      <td>0.000963</td>\n",
       "      <td>0.01</td>\n",
       "      <td>7</td>\n",
       "      <td>{'est__learning_rate': 0.01, 'est__max_depth': 7}</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.765</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.734</td>\n",
       "      <td>0.023324</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0       0.178265      0.008670         0.006912        0.001198   \n",
       "1       0.315358      0.017413         0.006609        0.001425   \n",
       "2       0.482575      0.019085         0.008350        0.000537   \n",
       "3       0.660290      0.010609         0.006561        0.001300   \n",
       "4       0.838601      0.033753         0.006765        0.000963   \n",
       "\n",
       "  param_est__learning_rate param_est__max_depth  \\\n",
       "0                     0.01                    3   \n",
       "1                     0.01                    4   \n",
       "2                     0.01                    5   \n",
       "3                     0.01                    6   \n",
       "4                     0.01                    7   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "0  {'est__learning_rate': 0.01, 'est__max_depth': 3}              0.705   \n",
       "1  {'est__learning_rate': 0.01, 'est__max_depth': 4}              0.730   \n",
       "2  {'est__learning_rate': 0.01, 'est__max_depth': 5}              0.725   \n",
       "3  {'est__learning_rate': 0.01, 'est__max_depth': 6}              0.730   \n",
       "4  {'est__learning_rate': 0.01, 'est__max_depth': 7}              0.755   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0              0.705              0.710              0.685              0.705   \n",
       "1              0.740              0.720              0.700              0.710   \n",
       "2              0.750              0.715              0.730              0.730   \n",
       "3              0.775              0.725              0.730              0.720   \n",
       "4              0.765              0.700              0.725              0.725   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "0            0.702        0.008718               70  \n",
       "1            0.720        0.014142               67  \n",
       "2            0.730        0.011402               60  \n",
       "3            0.736        0.019849               47  \n",
       "4            0.734        0.023324               52  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.DataFrame(gsearch.cv_results_)\n",
    "scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_est__learning_rate</th>\n",
       "      <th>param_est__max_depth</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.190787</td>\n",
       "      <td>0.010879</td>\n",
       "      <td>0.007576</td>\n",
       "      <td>0.000844</td>\n",
       "      <td>0.12</td>\n",
       "      <td>3</td>\n",
       "      <td>{'est__learning_rate': 0.12, 'est__max_depth': 3}</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.765</td>\n",
       "      <td>0.785</td>\n",
       "      <td>0.745</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.014142</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "7       0.190787      0.010879         0.007576        0.000844   \n",
       "\n",
       "  param_est__learning_rate param_est__max_depth  \\\n",
       "7                     0.12                    3   \n",
       "\n",
       "                                              params  split0_test_score  \\\n",
       "7  {'est__learning_rate': 0.12, 'est__max_depth': 3}              0.755   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "7              0.765              0.785              0.745               0.75   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "7             0.76        0.014142                1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores[scores.rank_test_score == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
